package zss.weibo.tqq;

import java.util.List;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

public class TencentWeiboPageProcessor implements PageProcessor {

	private Site site = new Site().setRetryTimes(3).setSleepTime(100)
			// 添加cookie之前一定要先设置主机地址，否则cookie信息不生效
			.setDomain("http://r.t.qq.com/cbdata/message/broadcast")
			// 添加抓包获取的cookie信息
			.addCookie("p_uin", "o826242827").addCookie("p_skey", "pZVwCmxQcM-PDLHJKjPcUdgw8-9iPMHn0jtokB1E7Uc_")
			// 添加请求头，有些网站会根据请求头判断该请求是由浏览器发起还是由爬虫发起的
			.addHeader("User-Agent", "AMB_612/061348&NA/061348&android7.0&6711214&FRDAL00&NA&NA&V3")
			.addHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
			.addHeader("Cookie", "p_uin=o826242827;p_skey=pZVwCmxQcM-PDLHJKjPcUdgw8-9iPMHn0jtokB1E7Uc_")
			.addHeader("Content-Length", "344")
			.addHeader("Connection", "keep-alive")
			.addHeader("Referer", "http://r.t.qq.com");

	// private Site site =
	// Site.me().setRetryTimes(3).setSleepTime(1000).setTimeOut(10000);

	public void process(Page page) {
		List<String> urls = page.getHtml().css("div#listWrapper").all();

		System.out.println(urls);
	}

	public Site getSite() {
		return site;
	}

	public static void main(String[] args) {
		Spider.create(new TencentWeiboPageProcessor())
				.addUrl("http://r.t.qq.com/cbdata/message/broadcast").thread(1)
				.run();
	}

}
